{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_761068/3844794198.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df.iloc[index]['content'] = '。'.join(sentences)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os, json\n",
    "\n",
    "# df = pd.read_csv('selected-ann.csv')\n",
    "df = pd.DataFrame(columns=['idd', 'content', 'score'])\n",
    "for idd in range(600):\n",
    "    filename = \"result/enhanced/{}-1.json\".format(idd)\n",
    "    if os.path.exists(filename):\n",
    "        content = json.load(open(filename, 'r', encoding='utf8'))['response']\n",
    "        score = int(idd/120)\n",
    "        # add to df\n",
    "        df.loc[len(df.index)] = [idd, content, score]\n",
    "\n",
    "stop_words = open('stopwords.txt', 'r', encoding='utf8').read().split('\\n')\n",
    "maxLen = 0\n",
    "maxSentences = []\n",
    "for index, row in df.iterrows():\n",
    "    content = row['content']\n",
    "    # 分句\n",
    "    sentences = content.split('。')\n",
    "    for stop_word in stop_words:\n",
    "        index = 0\n",
    "        for sentence in sentences:\n",
    "            if stop_word in sentence:\n",
    "                sentences[index] = sentence.replace(stop_word, '')\n",
    "            index += 1\n",
    "    df.iloc[index]['content'] = '。'.join(sentences)\n",
    "    if len(sentences) > maxLen:\n",
    "        maxLen = len(sentences)\n",
    "        maxSentences = sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/anaconda3/envs/deep2/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('./bert-base-chinese')\n",
    "bert_model = BertModel.from_pretrained('./bert-base-chinese')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "# 定义函数以获取BERT特征\n",
    "def get_bert_features(sentences):\n",
    "    MAX_SEQ_LEN = 33    \n",
    "    # 2. 使用分词器预处理文本\n",
    "    finalFeatures = [[0 for i in range(768)] for j in range(MAX_SEQ_LEN)]\n",
    "    index = 0\n",
    "    for text in sentences.split('。'):\n",
    "        inputs = tokenizer(text, return_tensors=\"pt\", max_length=512, truncation=True, padding='max_length')\n",
    "        with torch.no_grad():\n",
    "            # 3. 将文本传递给BERT模型\n",
    "            outputs = bert_model(**inputs)\n",
    "        # 取[CLS]标记的输出作为文本表示，返回一维数组\n",
    "        features = outputs.last_hidden_state[:, 0, :].numpy()\n",
    "        for i in range(0, len(features[0])):\n",
    "            finalFeatures[index][i] = features[0][i]\n",
    "        index += 1\n",
    "    return finalFeatures\n",
    "        \n",
    "\n",
    "# 将df中的内容列应用到get_bert_features函数\n",
    "df['bert_features'] = df['content'].apply(get_bert_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# 将df中的bert_features列转换为普通的列表\n",
    "bert_features = np.array(df['bert_features'].tolist())\n",
    "\n",
    "# 把df保存到本地tsv，不要使用科学计数法\n",
    "np.set_printoptions(suppress=True)\n",
    "# 保存df到本地，JSON格式\n",
    "df.to_json('selected-ann-bert-enhanced.json', orient='records', force_ascii=False, lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 4s 8ms/sample - loss: 3.0216\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.9744\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.6602\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5075\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5448\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4052\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5403\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2464\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0754\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0530\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8623\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8314\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6926\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8576\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6855\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6135\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5938\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4241\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5416\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5766\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4391\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3667\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2999\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2677\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2586\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2237\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2405\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2281\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2116\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1935\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1627\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1622\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1740\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1639\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1871\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1279\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1331\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1478\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1306\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1298\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1166\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1436\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2419\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1440\n",
      "Epoch 45/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1365\n",
      "Epoch 46/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1111\n",
      "Epoch 47/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1102\n",
      "Epoch 48/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1073\n",
      "Epoch 49/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1094\n",
      "Epoch 50/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0931\n",
      "Epoch 51/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0957\n",
      "Epoch 52/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1074\n",
      "Epoch 53/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1094\n",
      "Epoch 54/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0930\n",
      "Epoch 55/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0904\n",
      "Epoch 56/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0763\n",
      "Epoch 57/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0816\n",
      "Epoch 58/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1042\n",
      "Epoch 59/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0930\n",
      "Epoch 60/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0860\n",
      "Epoch 61/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.0875Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.0888\n",
      "Epoch 00061: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 4s 8ms/sample - loss: 2.8414\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.9309\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5455\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4756\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.3758\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.3948\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.9876\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0799\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.9279\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8205\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.7526\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6773\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6376\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5892\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5241\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4236\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5368\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5137\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4046\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3563\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3528\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5048\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4409\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3176\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2440\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2528\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2185\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2947\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2488\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2143\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1935\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2066\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2040\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1378\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2155\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1995\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1375\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1330\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1429\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1159\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1170\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1200\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2255\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2196\n",
      "Epoch 45/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.1888Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1849\n",
      "Epoch 00045: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 5s 10ms/sample - loss: 3.1425\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 2.0429\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.6174\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5357\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4271\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.3931\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2973\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0307\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0233\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.9716\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8729\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.9512\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8249\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.7430\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6843\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5994\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4872\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4725\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5704\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4994\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5303\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3863\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3035\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3009\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2979\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3003\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3417\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3268\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2433\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2489\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2075\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2184\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1936\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1865\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1887\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1597\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1540\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1471\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1564\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1565\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1351\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1410\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1848\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1682\n",
      "Epoch 45/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1331\n",
      "Epoch 46/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1427\n",
      "Epoch 47/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1436\n",
      "Epoch 48/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1871\n",
      "Epoch 49/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1791\n",
      "Epoch 50/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.1534Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1521\n",
      "Epoch 00050: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 4s 8ms/sample - loss: 2.9514\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.9232\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.5418\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4137\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.3718\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4692\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2005\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2458\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.1216\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.9350\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8823\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.7896\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8666\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8225\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.7189\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6534\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5605\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5568\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4448\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4199\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3251\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3436\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3727\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3214\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3279\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3559\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2157\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2400\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2235\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2001\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2041\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1818\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1506\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2127\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3010\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1966\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1786\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1468\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1701\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1371\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1407\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1343\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1932\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1489\n",
      "Epoch 45/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1182\n",
      "Epoch 46/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1243\n",
      "Epoch 47/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1340\n",
      "Epoch 48/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1318\n",
      "Epoch 49/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1306\n",
      "Epoch 50/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.1248Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1219\n",
      "Epoch 00050: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 4s 8ms/sample - loss: 3.0835\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 2.0390\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.7459\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.4520\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2802\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.2179\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0994\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.1931\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 1.0782\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8998\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.8651\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.7693\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6454\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.6374\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5157\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.5581\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4696\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.4522\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3243\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3032\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.3484\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2428\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2522\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2375\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2220\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2084\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2846\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2338\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2115\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.2094\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1554\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1562\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1707\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1370\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1378\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1169\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1381\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1404\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1637\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1406\n",
      "Epoch 41/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.1401Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 1s 2ms/sample - loss: 0.1389\n",
      "Epoch 00041: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 0s 536us/sample - loss: 7.6169\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 0s 145us/sample - loss: 2.0172\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 0s 148us/sample - loss: 1.2926\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 0s 150us/sample - loss: 1.0279\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 0s 158us/sample - loss: 0.8592\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 0s 141us/sample - loss: 0.6629\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 0s 145us/sample - loss: 0.5911\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.5086\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.3944\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.3139\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.2713\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 0s 141us/sample - loss: 0.2156\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.1950\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.1479\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.1262\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.1005\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0832\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0686\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0635\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0641\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0554\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0342\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0375\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0279\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.0267\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0376\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0313\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0364\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0376\n",
      "Epoch 30/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.0578Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0548\n",
      "Epoch 00030: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 0s 539us/sample - loss: 7.3637\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 0s 141us/sample - loss: 2.0155\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 0s 144us/sample - loss: 1.2972\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 0s 143us/sample - loss: 0.8949\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 0s 142us/sample - loss: 0.7102\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.5431\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.4099\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.3208\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.2757\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.2035\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.1927\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.1334\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.1115\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.1027\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.0873\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0766\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0853\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0628\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0630\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0494\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0688\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0763\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0574\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0456\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0348\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0308\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0303\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0260\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0239\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0224\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0161\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0214\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0174\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0197\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0164\n",
      "Epoch 36/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.0154Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.0173\n",
      "Epoch 00036: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 0s 545us/sample - loss: 7.3469\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 2.4413\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 1.3952\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 0s 144us/sample - loss: 1.0311\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 0s 142us/sample - loss: 0.8280\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.6622\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.5679\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.5069\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.3737\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 0s 125us/sample - loss: 0.3156\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.2712\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.2339\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.1795\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.1470\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 0s 124us/sample - loss: 0.1302\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0990\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0850\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.0785\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0632\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0563\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 0s 126us/sample - loss: 0.0500\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 0s 126us/sample - loss: 0.0448\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0384\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 0s 125us/sample - loss: 0.0412\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 0s 126us/sample - loss: 0.0406\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0366\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0272\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0301\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0450\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0457\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 0s 144us/sample - loss: 0.0380\n",
      "Epoch 32/100\n",
      "416/480 [=========================>....] - ETA: 0s - loss: 0.0306Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0296\n",
      "Epoch 00032: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 0s 540us/sample - loss: 9.7167\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 2.7855\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 1.8151\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 1.0576\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.8388\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.7691\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.5594\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.5557\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.3934\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.3295\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.2593\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.2242\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.1871\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 0s 141us/sample - loss: 0.1672\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.1177\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 0s 125us/sample - loss: 0.1089\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0997\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.0843\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.0882\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0978\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0850\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 0s 127us/sample - loss: 0.0687\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 0s 126us/sample - loss: 0.0635\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0562\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0484\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.0565\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0389\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0342\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0371\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0359\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0362\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0439\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0326\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0313\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0240\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0240\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0262\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0244\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0252\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 0s 128us/sample - loss: 0.0192\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.0252\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0233\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0272\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0177\n",
      "Epoch 45/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0200\n",
      "Epoch 46/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0177\n",
      "Epoch 47/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0148\n",
      "Epoch 48/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0238\n",
      "Epoch 49/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.0239\n",
      "Epoch 50/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0225\n",
      "Epoch 51/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0294\n",
      "Epoch 52/100\n",
      "416/480 [=========================>....] - ETA: 0s - loss: 0.0243Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0232\n",
      "Epoch 00052: early stopping\n",
      "Train on 480 samples\n",
      "Epoch 1/100\n",
      "480/480 [==============================] - 0s 543us/sample - loss: 8.9975\n",
      "Epoch 2/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 2.4678\n",
      "Epoch 3/100\n",
      "480/480 [==============================] - 0s 145us/sample - loss: 1.4628\n",
      "Epoch 4/100\n",
      "480/480 [==============================] - 0s 152us/sample - loss: 0.9791\n",
      "Epoch 5/100\n",
      "480/480 [==============================] - 0s 145us/sample - loss: 0.7603\n",
      "Epoch 6/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.6187\n",
      "Epoch 7/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.5090\n",
      "Epoch 8/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.4013\n",
      "Epoch 9/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.3196\n",
      "Epoch 10/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.2584\n",
      "Epoch 11/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.2314\n",
      "Epoch 12/100\n",
      "480/480 [==============================] - 0s 141us/sample - loss: 0.1818\n",
      "Epoch 13/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.1719\n",
      "Epoch 14/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.1340\n",
      "Epoch 15/100\n",
      "480/480 [==============================] - 0s 143us/sample - loss: 0.1088\n",
      "Epoch 16/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0905\n",
      "Epoch 17/100\n",
      "480/480 [==============================] - 0s 140us/sample - loss: 0.0871\n",
      "Epoch 18/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0674\n",
      "Epoch 19/100\n",
      "480/480 [==============================] - 0s 138us/sample - loss: 0.0609\n",
      "Epoch 20/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0610\n",
      "Epoch 21/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0599\n",
      "Epoch 22/100\n",
      "480/480 [==============================] - 0s 136us/sample - loss: 0.0515\n",
      "Epoch 23/100\n",
      "480/480 [==============================] - 0s 139us/sample - loss: 0.0546\n",
      "Epoch 24/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0397\n",
      "Epoch 25/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0418\n",
      "Epoch 26/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0392\n",
      "Epoch 27/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0421\n",
      "Epoch 28/100\n",
      "480/480 [==============================] - 0s 131us/sample - loss: 0.0345\n",
      "Epoch 29/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0279\n",
      "Epoch 30/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0289\n",
      "Epoch 31/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0271\n",
      "Epoch 32/100\n",
      "480/480 [==============================] - 0s 137us/sample - loss: 0.0191\n",
      "Epoch 33/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0212\n",
      "Epoch 34/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0200\n",
      "Epoch 35/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0174\n",
      "Epoch 36/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0166\n",
      "Epoch 37/100\n",
      "480/480 [==============================] - 0s 132us/sample - loss: 0.0180\n",
      "Epoch 38/100\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0163\n",
      "Epoch 39/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0158\n",
      "Epoch 40/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0130\n",
      "Epoch 41/100\n",
      "480/480 [==============================] - 0s 129us/sample - loss: 0.0178\n",
      "Epoch 42/100\n",
      "480/480 [==============================] - 0s 133us/sample - loss: 0.0233\n",
      "Epoch 43/100\n",
      "480/480 [==============================] - 0s 135us/sample - loss: 0.0244\n",
      "Epoch 44/100\n",
      "480/480 [==============================] - 0s 130us/sample - loss: 0.0229\n",
      "Epoch 45/100\n",
      "448/480 [===========================>..] - ETA: 0s - loss: 0.0208Restoring model weights from the end of the best epoch.\n",
      "480/480 [==============================] - 0s 134us/sample - loss: 0.0199\n",
      "Epoch 00045: early stopping\n",
      "Average QWK for LSTM: 0.6600753295668551\n",
      "Average QWK for CNN: 0.5914893617021277\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense, Conv1D, GlobalMaxPooling1D, Embedding, Dropout, Bidirectional, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "import ast\n",
    "\n",
    "# read df from json\n",
    "df = pd.read_json('selected-ann-bert.json', orient='records', lines=True)\n",
    "\n",
    "# 将vector_sequence转换为适当的形式\n",
    "X = np.stack(df['bert_features'].values)\n",
    "y = df['score'].values\n",
    "early_stopping = EarlyStopping(monitor='loss', patience=5, verbose=1, restore_best_weights=True)\n",
    "\n",
    "def get_lstm_model(input_shape):\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(LSTM(256, return_sequences=True), input_shape=input_shape))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Bidirectional(LSTM(128)))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1, activation='linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer=Adam(learning_rate=0.001))\n",
    "    return model\n",
    "\n",
    "def get_cnn_model(input_shape):\n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(128, 3, activation='relu', input_shape=input_shape))\n",
    "    model.add(GlobalMaxPooling1D())\n",
    "    model.add(Dense(1, activation='linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer=Adam(learning_rate=0.001))\n",
    "    return model\n",
    "\n",
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "def evaluate_model(model_func, model_name):\n",
    "    trues = []\n",
    "    preds = []\n",
    "    for train_index, val_index in skf.split(X, y):\n",
    "        X_train, X_val = X[train_index], X[val_index]\n",
    "        y_train, y_val = y[train_index], y[val_index]\n",
    "\n",
    "        model = model_func(X_train[0].shape)\n",
    "        model.fit(X_train, y_train, epochs=100, batch_size=32, verbose=1, callbacks=[early_stopping])\n",
    "        predictions = model.predict(X_val).flatten()\n",
    "        predictions = np.rint(predictions).astype(int)\n",
    "        trues = np.concatenate((trues, y_val))\n",
    "        preds = np.concatenate((preds, predictions))\n",
    "        \n",
    "    model.save(model_name + '.h5')\n",
    "    return cohen_kappa_score(trues, preds, weights='quadratic')\n",
    "\n",
    "lstm_qwk = evaluate_model(get_lstm_model, \"lstm\")\n",
    "cnn_qwk = evaluate_model(get_cnn_model, \"cnn\")\n",
    "\n",
    "print(f\"Average QWK for LSTM: {lstm_qwk}\")\n",
    "print(f\"Average QWK for CNN: {cnn_qwk}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jupyter",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
